Tensorflow¶
Keras Multi-layer Perceptron
Tensorflow Premade Estimators
Tensorflow Image and Text Classification
Tensorflow Boosted Trees Classifier with Feature Importance Analysis
In [1]:
import numpy as np
import pandas as pd
from sklearn import datasets
data = datasets.load_breast_cancer()
Data = pd.DataFrame(data['data'], columns = [x.title() for x in data['feature_names']])
Labels_dict = dict(zip(list(np.sort(np.unique(data['target'].tolist()))),
list([x.title() for x in data['target_names']])))
Target = 'Diagnosis'
Data[Target] = data['target']
# Data['Diagnosis'] = data['target'].replace(Labels_dict)
display(Data)
print(data['DESCR'])
Features with high variance¶
Moreover, high variance for some features can hurt our modeling process. For this reason, we would like to standardize features by removing the mean and scaling to unit variance.
In [2]:
from sklearn import preprocessing
X = Data.drop(columns = [Target])
y = Data[Target]
Temp = X.var().to_frame(name= 'Variance (Origial)').round(4)
scaler = preprocessing.StandardScaler()
X_std = scaler.fit_transform(X)
X_std = pd.DataFrame(data = X_std, columns = X.columns)
Temp = Temp.join(X_std.var().to_frame(name= 'Variance (Normalized)').round(4))
display(Temp.style.background_gradient(cmap='Reds', subset = 'Variance (Origial)')\
.background_gradient(cmap='Greens', subset = 'Variance (Normalized)'))
del Temp
Train and Test sets¶
In [3]:
import plotly.express as px
from HD_DeepLearning import DatasetTargetDist
Pull = [0 for x in range((len(Labels_dict)-1))]
Pull.append(.05)
PD = dict(PieColors = ['SeaGreen','FireBrick'], TableColors = ['Navy','White'], hole = .4,
column_widths=[0.6, 0.4], textfont = 14, height = 400, tablecolumnwidth = [0.25, 0.15, 0.15],
pull = Pull, legend_title = Target, title_x = 0.5, title_y = .9, pie_legend = [0.01, 0.01])
del Pull
DatasetTargetDist(Data, Target, Labels_dict, PD, orientation= 'columns')
StratifiedKFold is a variation of k-fold which returns stratified folds: each set contains approximately the same percentage of samples of each target class as the complete set.
In [4]:
from sklearn.model_selection import StratifiedShuffleSplit
def HD_StratifiedShuffleSplit(X, y, Test_Size = 0.3):
sss = StratifiedShuffleSplit(n_splits=1, test_size=Test_Size, random_state=42)
_ = sss.get_n_splits(X, y)
for train_index, test_index in sss.split(X, y):
# X
if isinstance(X, pd.DataFrame):
X_train, X_test = X.loc[train_index], X.loc[test_index]
else:
X_train, X_test = X[train_index], X[test_index]
# y
if isinstance(y, pd.Series):
y_train, y_test = y[train_index], y[test_index]
else:
y_train, y_test = y[train_index], y[test_index]
del sss
return X_train, y_train, X_test, y_test
X_train, y_train, X_test, y_test = HD_StratifiedShuffleSplit(X_std, y)
from HD_DeepLearning import Train_Test_Dist
PD.update(dict(column_widths=[0.3, 0.3, 0.3], tablecolumnwidth = [0.2, 0.4], height = 550, legend_title = Target))
Train_Test_Dist(X_train, y_train, X_test, y_test, PD, Labels_dict)
#
import tensorflow as tf
# y_train = tf.keras.utils.to_categorical(y_train, num_classes=len(Labels_dict))
# y_test = tf.keras.utils.to_categorical(y_test, num_classes=len(Labels_dict))
Feature Columns¶
Create the feature columns, using the original numeric columns as is and one-hot-encoding categorical variables.
In [5]:
def Feat_Columns(Inp, Numeric = False, disp_dtype = False):
'''
Feature Columns function
Input: Dataset
Output: Tensorflow Feature Column List
'''
if not Numeric:
Numeric = ['int64', 'int32', 'float64', 'float32']
Temp = Inp.dtypes.reset_index(drop = False)
Temp.columns = ['Features', 'Data Type']
Temp['Data Type'] = Temp['Data Type'].astype(str)
# Numeric_Columns
Numeric_Columns = Temp.loc[Temp['Data Type'].isin(Numeric), 'Features'].tolist()
# Categorical_Columns
# Categorical_Columns = Temp.loc[(~Temp['Data Type'].isin(Numeric)), 'Features'].tolist()
Categorical_Columns = Temp.loc[Temp['Data Type'] == 'object','Features'].tolist()
if disp_dtype:
display(pd.DataFrame({'Numeric Columns': [', '.join(Numeric_Columns)],
'Categorical Columns': [', '.join(Categorical_Columns)]}, index = ['Columns']).T.style)
# Feature Columns
feature_columns = []
if len(Categorical_Columns)>0:
for feature_name in Categorical_Columns:
vocabulary = Inp[feature_name].unique()
feature_columns.append(tf.feature_column.indicator_column(\
tf.feature_column.categorical_column_with_vocabulary_list(feature_name, vocabulary)))
if len(Numeric_Columns)>0:
for feature_name in Numeric_Columns:
feature_columns.append(tf.feature_column.numeric_column(feature_name))
return feature_columns
Input Function
The input_function defines how data is converted to a tf.data.Dataset that provides the input pipeline.
In [6]:
def make_input_fn(X, y, inmemory_train = False, n_epochs= None, shuffle=True, batch_size = 256):
# Not In memory Training
if not inmemory_train:
def input_fn():
dataset = tf.data.Dataset.from_tensor_slices((X.to_dict(orient='list'), y))
if shuffle:
dataset = dataset.shuffle(1000)
dataset = (dataset.repeat(n_epochs).batch(batch_size))
return dataset
# In memory Training
if inmemory_train:
y = np.expand_dims(y, axis=1)
def input_fn():
return dict(X), y
# End
return input_fn
Building the input pipeline¶
In [7]:
my_feature_columns = Feat_Columns(X)
# Training and evaluation input functions.
train_input_fn = make_input_fn(X_train, y_train)
eval_input_fn = make_input_fn(X_test, y_test, shuffle=False, n_epochs=1)
Modeling: Boosted Trees Classifier
In [8]:
from IPython.display import clear_output
from timeit import default_timer as timer
# Classifier
tf.keras.backend.clear_session()
IT = int(1e3)
params = {'n_trees': 50, 'max_depth': 3, 'n_batches_per_layer': 1, 'center_bias': True}
classifier = tf.estimator.BoostedTreesClassifier(my_feature_columns, **params)
# Train model.
start = timer()
classifier.train(train_input_fn, max_steps = IT)
CPU_Time = timer() - start
# Evaluation.
results = classifier.evaluate(eval_input_fn)
clear_output()
results['CPU Time'] = CPU_Time
display(pd.DataFrame(results, index = ['']).round(4))
ROC Curves¶
In [9]:
from HD_DeepLearning import ROC_Curve
# converting y_test to categorical
y_test_cat = tf.keras.utils.to_categorical(y_test, num_classes = len(Labels_dict), dtype='float32')
pred_dicts = list(classifier.predict(input_fn=eval_input_fn))
clear_output()
probs = np.array([pred['probabilities'] for pred in pred_dicts])
ROC_Curve(y_test_cat, probs, n_classes = len(Labels_dict), FS = 8)
Confusion Matrix¶
The confusion matrix allows for visualization of the performance of an algorithm. Note that due to the size of data, here we don't provide a Cross-validation evaluation. In general, this type of evaluation is preferred.
In [10]:
def Confusion_Mat(CM_Train, CM_Test, PD, n_splits = 10):
if n_splits == None:
Titles = ['Train Set', 'Test Set']
else:
Titles = ['Train Set (CV = % i)' % n_splits, 'Test Set (CV = % i)' % n_splits]
CM = [CM_Train, CM_Test]
Cmap = ['Greens', 'YlGn','Blues', 'PuBu']
for i in range(2):
fig, ax = plt.subplots(1, 2, figsize= PD['FS'])
fig.suptitle(Titles[i], weight = 'bold', fontsize = 16)
_ = sns.heatmap(CM[i], annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i], ax = ax[0],
linewidths = 0.2, cbar_kws={"shrink": PD['shrink']})
_ = ax[0].set_title('Confusion Matrix');
Temp = np.round(CM[i].astype('float') / CM[i].sum(axis=1)[:, np.newaxis], 2)
_ = sns.heatmap(Temp,
annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i+1], ax = ax[1],
linewidths = 0.4, vmin=0, vmax=1, cbar_kws={"shrink": PD['shrink']})
_ = ax[1].set_title('Normalized Confusion Matrix');
for a in ax:
_ = a.set_xlabel('Predicted labels')
_ = a.set_ylabel('True labels');
_ = a.xaxis.set_ticklabels(PD['Labels'])
_ = a.yaxis.set_ticklabels(PD['Labels'])
_ = a.set_aspect(1)
Train in Memory
An alternative way to train a model with boosting performance is using the train_in_memory feature. However, if there is no issue with performance or long training time is not a concern, training without this feature is recommended [2]. Furthermore, our observations have shown that using train_in_memory not always increases the performance of the training.
In [11]:
in_memory_params = dict(params)
in_memory_params['n_batches_per_layer'] = 1
# In-memory input_fn does not use batching.
train_input_fn = make_input_fn(X_train, y_train, inmemory_train = True)
# Classifier
tf.keras.backend.clear_session()
classifier = tf.estimator.BoostedTreesClassifier(my_feature_columns, train_in_memory=True, **in_memory_params)
# Train model.
start = timer()
classifier.train(train_input_fn, max_steps = IT)
CPU_Time = timer() - start
# Evaluation.
results = classifier.evaluate(eval_input_fn)
clear_output()
results['CPU Time'] = CPU_Time
display(pd.DataFrame(results, index = ['']).round(4))
ROC Curves¶
In [12]:
# converting y_test to categorical
y_test_cat = tf.keras.utils.to_categorical(y_test, num_classes = len(Labels_dict), dtype='float32')
pred_dicts = list(classifier.predict(input_fn=eval_input_fn))
clear_output()
probs = np.array([pred['probabilities'] for pred in pred_dicts])
ROC_Curve(y_test_cat, probs, n_classes = len(Labels_dict), FS = 8)
Feature Importance¶
We can investigate the feature importance of an artificial classification task. This is similar to that of scikit-learn and has been outlined in [6].
In [13]:
pred_dicts = list(classifier.experimental_predict_with_explanations(eval_input_fn))
clear_output()
# Create DFC Pandas dataframe.
labels = y_test
probs = pd.Series([pred['probabilities'][1] for pred in pred_dicts])
df_dfc = pd.DataFrame([pred['dfc'] for pred in pred_dicts])
df_dfc.columns = [x.replace('_',' ') for x in df_dfc.columns]
display(df_dfc.describe().T.style.background_gradient(subset= ['mean'], cmap='RdYlGn')\
.background_gradient(subset= ['std'], cmap='RdYlGn')\
.background_gradient(subset= ['min'], cmap='hot')\
.background_gradient(subset= ['max'], cmap='winter')
.format(precision=4).format({'count': "{:.0f}"}))
A nice property of DFCs is that the sum of the contributions + the bias is equal to the prediction for a given example.
In [14]:
# Sum of DFCs + bias == probabality.
bias = pred_dicts[0]['bias']
dfc_prob = df_dfc.sum(axis=1) + bias
np.testing.assert_almost_equal(dfc_prob.values, probs.values)
Feature Importance for a patient¶
Plot DFCs for an individual patient which is color-coded based on the contributions' directionality and add the feature values on the figure.
In [15]:
import matplotlib.pyplot as plt
from matplotlib.font_manager import FontProperties
def _add_feature_values(feature_values, ax, colors):
"""Display feature's values on left of plot."""
x_coord = ax.get_xlim()[0]
OFFSET = 0.15
for y_coord, (feat_name, feat_val) in enumerate(feature_values.items()):
t = plt.text(x_coord, y_coord - OFFSET, '{}'.format(feat_val), size=12)
t.set_bbox(dict(facecolor= colors[y_coord], alpha=0.25))
font = FontProperties()
# font.set_weight('bold')
t = plt.text(x_coord, y_coord + 1 - OFFSET, 'Feature\nValue', fontproperties=font, size=13)
def _xLims(ax):
Temp = np.linspace(-1,1,21, endpoint=True)
Temp = np.round(Temp,1)
xlims = ax.get_xlim()
for l, r in list(zip(Temp[:-1],Temp[1:])):
if l<= xlims[0] < r:
Left = l
if l<= xlims[1] < r:
Right = r
return [Left, Right]
def Plot_Example(example, TOP_N = 10, Pos_Color = 'LimeGreen', Neg_Color = 'OrangeRed', Maps = None, FS = (13, 7)):
# Sorting by absolute value
sorted_ix = example.abs().sort_values()[-TOP_N:].index
example = example[sorted_ix]
fig, ax = plt.subplots(1, 1, figsize= FS)
Temp = example.to_frame('Value').sort_index(ascending= False)
Temp0 = Temp.copy(); Temp0[Temp0 < 0] = np.nan
_ = Temp0.plot(kind='barh', color= Pos_Color, edgecolor = 'white', hatch = '///', legend=None, alpha=0.75, ax = ax)
Temp0 = Temp.copy(); Temp0[Temp0 >= 0] = np.nan
_ = Temp0.plot(kind='barh', color= Neg_Color, edgecolor = 'white', hatch = '///', legend=None, alpha=0.75, ax = ax)
_ = Temp.plot(kind='barh', color='None', edgecolor = 'Black', legend=None, alpha=1, lw=1.2, ax = ax)
del Temp, Temp0
_ = ax.grid(False, axis='y')
# x axis Limits
_ = ax.set_xlim(_xLims(ax))
# Add feature values.
Temp = X_test.copy()
Temp.columns = [x.replace('_',' ') for x in Temp.columns]
if not Maps == None:
for c in Maps.keys():
Temp[c] = Temp[c].map(Maps[c])
colors = example.map(lambda x: Pos_Color if x >= 0 else Neg_Color).tolist()
_add_feature_values(Temp.iloc[ID][sorted_ix].round(4), ax, colors)
return ax
In [16]:
ID = 61
Tops = X_train.shape[1]
ax = Plot_Example(df_dfc.iloc[ID], TOP_N = Tops, FS = (13, 16))
_ = ax.set_title('Feature contributions for example patient {} from the Test set\n Pred: {:1.2f}; Label: {}'
.format(ID, probs[ID], labels.iloc[ID]))
_ = ax.set_xlabel('Contribution to Predicted Probability', size=14)
References¶
Detrano, R., Janosi, A., Steinbrunn, W., Pfisterer, M., Schmid, J.J., Sandhu, S., Guppy, K.H., Lee, S. and Froelicher, V., 1989. International application of a new probability algorithm for the diagnosis of coronary artery disease. The American journal of cardiology, 64(5), pp.304-310.
Aha, D. and Kibler, D., 1988. Instance-based prediction of heart-disease presence with the Cleveland database. University of California, 3(1), pp.3-2.
Gennari, J.H., Langley, P. and Fisher, D., 1989. Models of incremental concept formation. Artificial intelligence, 40(1-3), pp.11-61.
- Regression analysis Wikipedia page
- Tensorflow tutorials
- TensorFlow Boosted Trees Classifier
- Lasso (statistics) Wikipedia page)
- Tikhonov regularizationm Wikipedia page
- Palczewska A., Palczewski J., Marchese Robinson R., Neagu D. (2014) Interpreting Random Forest Classification Models Using a Feature Contribution Method. In: Bouabana-Tebibel T., Rubin S. (eds) Integration of Reusable Systems. Advances in Intelligent Systems and Computing, vol 263. Springer, Cham
- S. Aeberhard, D. Coomans and O. de Vel, Comparison of Classifiers in High Dimensional Settings, Tech. Rep. no. 92-02, (1992), Dept. of Computer Science and Dept. of Mathematics and Statistics, James Cook University of North Queensland. (Also submitted to Technometrics).
- S. Aeberhard, D. Coomans and O. de Vel, “THE CLASSIFICATION PERFORMANCE OF RDA” Tech. Rep. no. 92-01, (1992), Dept. of Computer Science and Dept. of Mathematics and Statistics, James Cook University of North Queensland. (Also submitted to Journal of Chemometrics).